{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Please input your directory for the top level folder\n",
    "folder name : SUBMISSION MODEL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "dir_ = 'INPUT-PROJECT-DIRECTORY/submission_model/' # input only here"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### setting other directory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "raw_data_dir = dir_+'2. data/'\n",
    "processed_data_dir = dir_+'2. data/processed/'\n",
    "log_dir = dir_+'4. logs/'\n",
    "model_dir = dir_+'5. models/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "####################################################################################\n",
    "##################### 1-3. recursive model by store & dept #########################\n",
    "####################################################################################"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "ver, KKK = 'priv', 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "STORES = ['CA_1', 'CA_2', 'CA_3', 'CA_4', 'TX_1', 'TX_2', 'TX_3', 'WI_1', 'WI_2', 'WI_3']\n",
    "DEPTS = ['HOBBIES_1', 'HOBBIES_2', 'HOUSEHOLD_1', 'HOUSEHOLD_2', 'FOODS_1', 'FOODS_2', 'FOODS_3']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5"
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import os, sys, gc, time, warnings, pickle, psutil, random\n",
    "\n",
    "from multiprocessing import Pool\n",
    "\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "########################### Helpers\n",
    "#################################################################################\n",
    "## Seeder\n",
    "def seed_everything(seed=0):\n",
    "    random.seed(seed)\n",
    "    np.random.seed(seed)\n",
    "\n",
    "    \n",
    "## Multiprocess Runs\n",
    "def df_parallelize_run(func, t_split):\n",
    "    num_cores = np.min([N_CORES,len(t_split)])\n",
    "    pool = Pool(num_cores)\n",
    "    df = pd.concat(pool.map(func, t_split), axis=1)\n",
    "    pool.close()\n",
    "    pool.join()\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "########################### Helper to load data by store ID\n",
    "#################################################################################\n",
    "# Read data\n",
    "def get_data_by_store(store, dept):\n",
    "    \n",
    "    # Read and contact basic feature\n",
    "    df = pd.concat([pd.read_pickle(BASE),\n",
    "                    pd.read_pickle(PRICE).iloc[:,2:],\n",
    "                    pd.read_pickle(CALENDAR).iloc[:,2:]],\n",
    "                    axis=1)\n",
    "    \n",
    "    df = df[df['d']>=START_TRAIN]\n",
    "    \n",
    "    df = df[(df['store_id']==store) & (df['dept_id']==dept)]\n",
    "\n",
    "    df2 = pd.read_pickle(MEAN_ENC)[mean_features]\n",
    "    df2 = df2[df2.index.isin(df.index)]\n",
    "        \n",
    "    df3 = pd.read_pickle(LAGS).iloc[:,3:]\n",
    "    df3 = df3[df3.index.isin(df.index)]\n",
    "    \n",
    "    df = pd.concat([df, df2], axis=1)\n",
    "    del df2\n",
    "    \n",
    "    df = pd.concat([df, df3], axis=1)\n",
    "    del df3\n",
    "    \n",
    "    features = [col for col in list(df) if col not in remove_features]\n",
    "    df = df[['id','d',TARGET]+features]\n",
    "    \n",
    "    df = df.reset_index(drop=True)\n",
    "    \n",
    "    return df, features\n",
    "\n",
    "# Recombine Test set after training\n",
    "def get_base_test():\n",
    "    base_test = pd.DataFrame()\n",
    "\n",
    "    for store_id in STORES:\n",
    "        for state_id in DEPTS:\n",
    "            temp_df = pd.read_pickle(processed_data_dir+'test_'+store_id+'_'+state_id+'.pkl')\n",
    "            temp_df['store_id'] = store_id\n",
    "            temp_df['dept_id'] = state_id\n",
    "            base_test = pd.concat([base_test, temp_df]).reset_index(drop=True)\n",
    "    \n",
    "    return base_test\n",
    "\n",
    "\n",
    "########################### Helper to make dynamic rolling lags\n",
    "#################################################################################\n",
    "def make_lag(LAG_DAY):\n",
    "    lag_df = base_test[['id','d',TARGET]]\n",
    "    col_name = 'sales_lag_'+str(LAG_DAY)\n",
    "    lag_df[col_name] = lag_df.groupby(['id'])[TARGET].transform(lambda x: x.shift(LAG_DAY)).astype(np.float16)\n",
    "    return lag_df[[col_name]]\n",
    "\n",
    "\n",
    "def make_lag_roll(LAG_DAY):\n",
    "    shift_day = LAG_DAY[0]\n",
    "    roll_wind = LAG_DAY[1]\n",
    "    lag_df = base_test[['id','d',TARGET]]\n",
    "    col_name = 'rolling_mean_tmp_'+str(shift_day)+'_'+str(roll_wind)\n",
    "    lag_df[col_name] = lag_df.groupby(['id'])[TARGET].transform(lambda x: x.shift(shift_day).rolling(roll_wind).mean())\n",
    "    return lag_df[[col_name]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "########################### Model params\n",
    "#################################################################################\n",
    "import lightgbm as lgb\n",
    "lgb_params = {\n",
    "                    'boosting_type': 'gbdt',\n",
    "                    'objective': 'tweedie',\n",
    "                    'tweedie_variance_power': 1.1,\n",
    "                    'metric': 'rmse',\n",
    "                    'subsample': 0.5,\n",
    "                    'subsample_freq': 1,\n",
    "                    'learning_rate': 0.015,\n",
    "                    'num_leaves': 2**8-1,\n",
    "                    'min_data_in_leaf': 2**8-1,\n",
    "                    'feature_fraction': 0.5,\n",
    "                    'max_bin': 100,\n",
    "                    'n_estimators': 3000,\n",
    "                    'boost_from_average': False,\n",
    "                    'verbose': -1\n",
    "                } "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "########################### Vars\n",
    "#################################################################################\n",
    "VER = 1                          \n",
    "SEED = 42                        \n",
    "seed_everything(SEED)            \n",
    "lgb_params['seed'] = SEED        \n",
    "N_CORES = psutil.cpu_count()     \n",
    "\n",
    "\n",
    "#LIMITS and const\n",
    "TARGET      = 'sales'            \n",
    "START_TRAIN = 700                \n",
    "END_TRAIN   = 1941 - 28*KKK      \n",
    "P_HORIZON   = 28                 \n",
    "USE_AUX     = False             \n",
    "\n",
    "remove_features = ['id','cat_id', 'state_id','store_id','dept_id',\n",
    "                   'date','wm_yr_wk','d',TARGET]\n",
    "mean_features   = ['enc_item_id_store_id_mean','enc_item_id_store_id_std'] \n",
    "\n",
    "ORIGINAL = raw_data_dir\n",
    "BASE     = processed_data_dir+'grid_part_1.pkl'\n",
    "PRICE    = processed_data_dir+'grid_part_2.pkl'\n",
    "CALENDAR = processed_data_dir+'grid_part_3.pkl'\n",
    "LAGS     = processed_data_dir+'lags_df_28.pkl'\n",
    "MEAN_ENC = processed_data_dir+'mean_encoding_df.pkl'\n",
    "\n",
    "\n",
    "#SPLITS for lags creation\n",
    "SHIFT_DAY  = 28\n",
    "N_LAGS     = 15\n",
    "LAGS_SPLIT = [col for col in range(SHIFT_DAY,SHIFT_DAY+N_LAGS)]\n",
    "ROLS_SPLIT = []\n",
    "for i in [1,7,14]:\n",
    "    for j in [7,14,30,60]:\n",
    "        ROLS_SPLIT.append([i,j])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train CA_1 HOBBIES_1\n",
      "[LightGBM] [Warning] feature_fraction is set=0.5, colsample_bytree=1.0 will be ignored. Current value: feature_fraction=0.5\n",
      "[LightGBM] [Warning] min_data_in_leaf is set=255, min_child_samples=20 will be ignored. Current value: min_data_in_leaf=255\n",
      "[100]\tvalid_0's rmse: 2.47656\n",
      "[200]\tvalid_0's rmse: 2.38003\n",
      "[300]\tvalid_0's rmse: 2.33057\n",
      "[400]\tvalid_0's rmse: 2.28083\n",
      "[500]\tvalid_0's rmse: 2.23399\n",
      "[600]\tvalid_0's rmse: 2.19142\n",
      "[700]\tvalid_0's rmse: 2.14641\n",
      "[800]\tvalid_0's rmse: 2.10179\n",
      "[900]\tvalid_0's rmse: 2.05978\n",
      "[1000]\tvalid_0's rmse: 2.0185\n",
      "[1100]\tvalid_0's rmse: 1.97704\n",
      "[1200]\tvalid_0's rmse: 1.9376\n",
      "[1300]\tvalid_0's rmse: 1.90014\n",
      "[1400]\tvalid_0's rmse: 1.8604\n",
      "[1500]\tvalid_0's rmse: 1.8219\n",
      "[1600]\tvalid_0's rmse: 1.78732\n",
      "[1700]\tvalid_0's rmse: 1.74923\n",
      "[1800]\tvalid_0's rmse: 1.7204\n",
      "[1900]\tvalid_0's rmse: 1.68947\n",
      "[2000]\tvalid_0's rmse: 1.65802\n",
      "[2100]\tvalid_0's rmse: 1.62881\n",
      "[2200]\tvalid_0's rmse: 1.60307\n",
      "[2300]\tvalid_0's rmse: 1.57616\n",
      "[2400]\tvalid_0's rmse: 1.55119\n",
      "[2500]\tvalid_0's rmse: 1.52511\n",
      "[2600]\tvalid_0's rmse: 1.50111\n",
      "[2700]\tvalid_0's rmse: 1.47917\n",
      "[2800]\tvalid_0's rmse: 1.4567\n",
      "[2900]\tvalid_0's rmse: 1.43514\n",
      "[3000]\tvalid_0's rmse: 1.41548\n",
      "Train CA_1 HOBBIES_2\n",
      "[LightGBM] [Warning] feature_fraction is set=0.5, colsample_bytree=1.0 will be ignored. Current value: feature_fraction=0.5\n",
      "[LightGBM] [Warning] min_data_in_leaf is set=255, min_child_samples=20 will be ignored. Current value: min_data_in_leaf=255\n",
      "[100]\tvalid_0's rmse: 0.770895\n",
      "[200]\tvalid_0's rmse: 0.753858\n",
      "[300]\tvalid_0's rmse: 0.741874\n",
      "[400]\tvalid_0's rmse: 0.727688\n",
      "[500]\tvalid_0's rmse: 0.712238\n",
      "[600]\tvalid_0's rmse: 0.697793\n",
      "[700]\tvalid_0's rmse: 0.680989\n",
      "[800]\tvalid_0's rmse: 0.664495\n",
      "[900]\tvalid_0's rmse: 0.647381\n",
      "[1000]\tvalid_0's rmse: 0.631491\n",
      "[1100]\tvalid_0's rmse: 0.615253\n",
      "[1200]\tvalid_0's rmse: 0.597171\n",
      "[1300]\tvalid_0's rmse: 0.583368\n",
      "[1400]\tvalid_0's rmse: 0.568539\n",
      "[1500]\tvalid_0's rmse: 0.553875\n",
      "[1600]\tvalid_0's rmse: 0.540909\n",
      "[1700]\tvalid_0's rmse: 0.526658\n",
      "[1800]\tvalid_0's rmse: 0.515589\n",
      "[1900]\tvalid_0's rmse: 0.502699\n",
      "[2000]\tvalid_0's rmse: 0.491302\n",
      "[2100]\tvalid_0's rmse: 0.480902\n",
      "[2200]\tvalid_0's rmse: 0.469546\n",
      "[2300]\tvalid_0's rmse: 0.458875\n",
      "[2400]\tvalid_0's rmse: 0.451133\n",
      "[2500]\tvalid_0's rmse: 0.44369\n",
      "[2600]\tvalid_0's rmse: 0.435933\n",
      "[2700]\tvalid_0's rmse: 0.427728\n",
      "[2800]\tvalid_0's rmse: 0.420482\n",
      "[2900]\tvalid_0's rmse: 0.413034\n",
      "[3000]\tvalid_0's rmse: 0.406685\n",
      "Train CA_1 HOUSEHOLD_1\n",
      "[LightGBM] [Warning] feature_fraction is set=0.5, colsample_bytree=1.0 will be ignored. Current value: feature_fraction=0.5\n",
      "[LightGBM] [Warning] min_data_in_leaf is set=255, min_child_samples=20 will be ignored. Current value: min_data_in_leaf=255\n",
      "[100]\tvalid_0's rmse: 1.582\n",
      "[200]\tvalid_0's rmse: 1.49982\n",
      "[300]\tvalid_0's rmse: 1.48064\n",
      "[400]\tvalid_0's rmse: 1.46468\n",
      "[500]\tvalid_0's rmse: 1.44878\n",
      "[600]\tvalid_0's rmse: 1.43472\n",
      "[700]\tvalid_0's rmse: 1.4237\n",
      "[800]\tvalid_0's rmse: 1.41126\n",
      "[900]\tvalid_0's rmse: 1.40143\n",
      "[1000]\tvalid_0's rmse: 1.39096\n",
      "[1100]\tvalid_0's rmse: 1.38075\n",
      "[1200]\tvalid_0's rmse: 1.37055\n",
      "[1300]\tvalid_0's rmse: 1.36125\n",
      "[1400]\tvalid_0's rmse: 1.35277\n",
      "[1500]\tvalid_0's rmse: 1.34331\n",
      "[1600]\tvalid_0's rmse: 1.33515\n",
      "[1700]\tvalid_0's rmse: 1.3272\n",
      "[1800]\tvalid_0's rmse: 1.31888\n",
      "[1900]\tvalid_0's rmse: 1.3105\n",
      "[2000]\tvalid_0's rmse: 1.30249\n",
      "[2100]\tvalid_0's rmse: 1.2957\n",
      "[2200]\tvalid_0's rmse: 1.2884\n",
      "[2300]\tvalid_0's rmse: 1.28073\n",
      "[2400]\tvalid_0's rmse: 1.27294\n",
      "[2500]\tvalid_0's rmse: 1.266\n",
      "[2600]\tvalid_0's rmse: 1.25854\n",
      "[2700]\tvalid_0's rmse: 1.25096\n",
      "[2800]\tvalid_0's rmse: 1.24479\n",
      "[2900]\tvalid_0's rmse: 1.23783\n",
      "[3000]\tvalid_0's rmse: 1.23128\n",
      "Train CA_1 HOUSEHOLD_2\n",
      "[LightGBM] [Warning] feature_fraction is set=0.5, colsample_bytree=1.0 will be ignored. Current value: feature_fraction=0.5\n",
      "[LightGBM] [Warning] min_data_in_leaf is set=255, min_child_samples=20 will be ignored. Current value: min_data_in_leaf=255\n",
      "[100]\tvalid_0's rmse: 0.758355\n",
      "[200]\tvalid_0's rmse: 0.73978\n",
      "[300]\tvalid_0's rmse: 0.734449\n",
      "[400]\tvalid_0's rmse: 0.728872\n",
      "[500]\tvalid_0's rmse: 0.723753\n",
      "[600]\tvalid_0's rmse: 0.718473\n",
      "[700]\tvalid_0's rmse: 0.713136\n",
      "[800]\tvalid_0's rmse: 0.707969\n",
      "[900]\tvalid_0's rmse: 0.703118\n",
      "[1000]\tvalid_0's rmse: 0.69784\n",
      "[1100]\tvalid_0's rmse: 0.693187\n",
      "[1200]\tvalid_0's rmse: 0.688798\n",
      "[1300]\tvalid_0's rmse: 0.683968\n",
      "[1400]\tvalid_0's rmse: 0.679905\n",
      "[1500]\tvalid_0's rmse: 0.675777\n",
      "[1600]\tvalid_0's rmse: 0.671209\n",
      "[1700]\tvalid_0's rmse: 0.666972\n",
      "[1800]\tvalid_0's rmse: 0.662967\n",
      "[1900]\tvalid_0's rmse: 0.658817\n",
      "[2000]\tvalid_0's rmse: 0.654388\n",
      "[2100]\tvalid_0's rmse: 0.650538\n",
      "[2200]\tvalid_0's rmse: 0.646885\n",
      "[2300]\tvalid_0's rmse: 0.642946\n",
      "[2400]\tvalid_0's rmse: 0.639566\n",
      "[2500]\tvalid_0's rmse: 0.635806\n",
      "[2600]\tvalid_0's rmse: 0.632039\n",
      "[2700]\tvalid_0's rmse: 0.62818\n",
      "[2800]\tvalid_0's rmse: 0.624301\n",
      "[2900]\tvalid_0's rmse: 0.620805\n",
      "[3000]\tvalid_0's rmse: 0.617296\n",
      "Train CA_1 FOODS_1\n",
      "[LightGBM] [Warning] feature_fraction is set=0.5, colsample_bytree=1.0 will be ignored. Current value: feature_fraction=0.5\n",
      "[LightGBM] [Warning] min_data_in_leaf is set=255, min_child_samples=20 will be ignored. Current value: min_data_in_leaf=255\n",
      "[100]\tvalid_0's rmse: 2.19938\n",
      "[200]\tvalid_0's rmse: 2.13065\n",
      "[300]\tvalid_0's rmse: 2.09606\n",
      "[400]\tvalid_0's rmse: 2.06338\n",
      "[500]\tvalid_0's rmse: 2.03189\n",
      "[600]\tvalid_0's rmse: 2.00095\n",
      "[700]\tvalid_0's rmse: 1.97021\n",
      "[800]\tvalid_0's rmse: 1.93744\n",
      "[900]\tvalid_0's rmse: 1.90598\n",
      "[1000]\tvalid_0's rmse: 1.87037\n",
      "[1100]\tvalid_0's rmse: 1.83616\n",
      "[1200]\tvalid_0's rmse: 1.80475\n",
      "[1300]\tvalid_0's rmse: 1.76703\n",
      "[1400]\tvalid_0's rmse: 1.72992\n",
      "[1500]\tvalid_0's rmse: 1.68765\n",
      "[1600]\tvalid_0's rmse: 1.64605\n",
      "[1700]\tvalid_0's rmse: 1.61158\n",
      "[1800]\tvalid_0's rmse: 1.57541\n",
      "[1900]\tvalid_0's rmse: 1.54178\n",
      "[2000]\tvalid_0's rmse: 1.50709\n",
      "[2100]\tvalid_0's rmse: 1.47251\n",
      "[2200]\tvalid_0's rmse: 1.44741\n",
      "[2300]\tvalid_0's rmse: 1.4211\n",
      "[2400]\tvalid_0's rmse: 1.40085\n",
      "[2500]\tvalid_0's rmse: 1.37717\n",
      "[2600]\tvalid_0's rmse: 1.35674\n",
      "[2700]\tvalid_0's rmse: 1.33764\n",
      "[2800]\tvalid_0's rmse: 1.31788\n",
      "[2900]\tvalid_0's rmse: 1.30027\n",
      "[3000]\tvalid_0's rmse: 1.28509\n",
      "Train CA_1 FOODS_2\n",
      "[LightGBM] [Warning] feature_fraction is set=0.5, colsample_bytree=1.0 will be ignored. Current value: feature_fraction=0.5\n",
      "[LightGBM] [Warning] min_data_in_leaf is set=255, min_child_samples=20 will be ignored. Current value: min_data_in_leaf=255\n",
      "[100]\tvalid_0's rmse: 1.69085\n",
      "[200]\tvalid_0's rmse: 1.58058\n",
      "[300]\tvalid_0's rmse: 1.55259\n",
      "[400]\tvalid_0's rmse: 1.53322\n",
      "[500]\tvalid_0's rmse: 1.51629\n",
      "[600]\tvalid_0's rmse: 1.50089\n",
      "[700]\tvalid_0's rmse: 1.48514\n",
      "[800]\tvalid_0's rmse: 1.47257\n",
      "[900]\tvalid_0's rmse: 1.45968\n",
      "[1000]\tvalid_0's rmse: 1.44646\n",
      "[1100]\tvalid_0's rmse: 1.43425\n",
      "[1200]\tvalid_0's rmse: 1.42247\n",
      "[1300]\tvalid_0's rmse: 1.4104\n",
      "[1400]\tvalid_0's rmse: 1.39875\n",
      "[1500]\tvalid_0's rmse: 1.38737\n",
      "[1600]\tvalid_0's rmse: 1.37553\n",
      "[1700]\tvalid_0's rmse: 1.36511\n",
      "[1800]\tvalid_0's rmse: 1.35396\n",
      "[1900]\tvalid_0's rmse: 1.34406\n",
      "[2000]\tvalid_0's rmse: 1.33535\n",
      "[2100]\tvalid_0's rmse: 1.32548\n",
      "[2200]\tvalid_0's rmse: 1.31558\n",
      "[2300]\tvalid_0's rmse: 1.30549\n",
      "[2400]\tvalid_0's rmse: 1.29512\n",
      "[2500]\tvalid_0's rmse: 1.28573\n",
      "[2600]\tvalid_0's rmse: 1.27653\n",
      "[2700]\tvalid_0's rmse: 1.26699\n",
      "[2800]\tvalid_0's rmse: 1.25737\n",
      "[2900]\tvalid_0's rmse: 1.24796\n",
      "[3000]\tvalid_0's rmse: 1.23843\n",
      "Train CA_1 FOODS_3\n",
      "[LightGBM] [Warning] feature_fraction is set=0.5, colsample_bytree=1.0 will be ignored. Current value: feature_fraction=0.5\n",
      "[LightGBM] [Warning] min_data_in_leaf is set=255, min_child_samples=20 will be ignored. Current value: min_data_in_leaf=255\n",
      "[100]\tvalid_0's rmse: 3.09194\n",
      "[200]\tvalid_0's rmse: 2.61573\n",
      "[300]\tvalid_0's rmse: 2.55001\n",
      "[400]\tvalid_0's rmse: 2.51633\n",
      "[500]\tvalid_0's rmse: 2.48974\n",
      "[600]\tvalid_0's rmse: 2.4632\n",
      "[700]\tvalid_0's rmse: 2.43934\n",
      "[800]\tvalid_0's rmse: 2.41994\n",
      "[900]\tvalid_0's rmse: 2.39994\n",
      "[1000]\tvalid_0's rmse: 2.3826\n",
      "[1100]\tvalid_0's rmse: 2.36845\n",
      "[1200]\tvalid_0's rmse: 2.35433\n",
      "[1300]\tvalid_0's rmse: 2.34051\n",
      "[1400]\tvalid_0's rmse: 2.32609\n",
      "[1500]\tvalid_0's rmse: 2.31252\n",
      "[1600]\tvalid_0's rmse: 2.29892\n",
      "[1700]\tvalid_0's rmse: 2.28444\n",
      "[1800]\tvalid_0's rmse: 2.27009\n",
      "[1900]\tvalid_0's rmse: 2.25739\n",
      "[2000]\tvalid_0's rmse: 2.24546\n",
      "[2100]\tvalid_0's rmse: 2.23173\n",
      "[2200]\tvalid_0's rmse: 2.22035\n",
      "[2300]\tvalid_0's rmse: 2.20813\n",
      "[2400]\tvalid_0's rmse: 2.19526\n",
      "[2500]\tvalid_0's rmse: 2.1836\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2600]\tvalid_0's rmse: 2.17212\n",
      "[2700]\tvalid_0's rmse: 2.16059\n",
      "[2800]\tvalid_0's rmse: 2.1489\n",
      "[2900]\tvalid_0's rmse: 2.13612\n",
      "[3000]\tvalid_0's rmse: 2.12609\n",
      "[1200]\tvalid_0's rmse: 1.48757\n",
      "[1300]\tvalid_0's rmse: 1.4564\n",
      "[1400]\tvalid_0's rmse: 1.42756\n",
      "[1500]\tvalid_0's rmse: 1.39612\n",
      "[1600]\tvalid_0's rmse: 1.36916\n",
      "[1700]\tvalid_0's rmse: 1.34183\n",
      "[1800]\tvalid_0's rmse: 1.31879\n",
      "[1900]\tvalid_0's rmse: 1.29443\n",
      "[2000]\tvalid_0's rmse: 1.27116\n",
      "[2100]\tvalid_0's rmse: 1.25035\n",
      "[2200]\tvalid_0's rmse: 1.23009\n",
      "[2300]\tvalid_0's rmse: 1.21117\n",
      "[2400]\tvalid_0's rmse: 1.19343\n",
      "[2500]\tvalid_0's rmse: 1.17467\n",
      "[2600]\tvalid_0's rmse: 1.15862\n",
      "[2700]\tvalid_0's rmse: 1.14196\n",
      "[2800]\tvalid_0's rmse: 1.123\n",
      "[2900]\tvalid_0's rmse: 1.1061\n",
      "[3000]\tvalid_0's rmse: 1.09407\n",
      "Train CA_2 HOBBIES_2\n",
      "[LightGBM] [Warning] feature_fraction is set=0.5, colsample_bytree=1.0 will be ignored. Current value: feature_fraction=0.5\n",
      "[LightGBM] [Warning] min_data_in_leaf is set=255, min_child_samples=20 will be ignored. Current value: min_data_in_leaf=255\n",
      "[100]\tvalid_0's rmse: 0.821965\n",
      "[200]\tvalid_0's rmse: 0.801632\n",
      "[300]\tvalid_0's rmse: 0.787814\n",
      "[400]\tvalid_0's rmse: 0.77205\n",
      "[500]\tvalid_0's rmse: 0.756334\n",
      "[600]\tvalid_0's rmse: 0.739908\n",
      "[700]\tvalid_0's rmse: 0.724724\n",
      "[800]\tvalid_0's rmse: 0.707868\n",
      "[900]\tvalid_0's rmse: 0.693476\n",
      "[1000]\tvalid_0's rmse: 0.675852\n",
      "[1100]\tvalid_0's rmse: 0.658528\n",
      "[1200]\tvalid_0's rmse: 0.641045\n",
      "[1300]\tvalid_0's rmse: 0.626629\n",
      "[1400]\tvalid_0's rmse: 0.611729\n",
      "[1500]\tvalid_0's rmse: 0.598469\n",
      "[1600]\tvalid_0's rmse: 0.584474\n",
      "[1700]\tvalid_0's rmse: 0.571244\n",
      "[1800]\tvalid_0's rmse: 0.559027\n",
      "[1900]\tvalid_0's rmse: 0.548309\n",
      "[2000]\tvalid_0's rmse: 0.536833\n",
      "[2100]\tvalid_0's rmse: 0.527209\n",
      "[2200]\tvalid_0's rmse: 0.515997\n",
      "[2300]\tvalid_0's rmse: 0.505853\n",
      "[2400]\tvalid_0's rmse: 0.496157\n",
      "[2500]\tvalid_0's rmse: 0.487528\n",
      "[2600]\tvalid_0's rmse: 0.478452\n",
      "[2700]\tvalid_0's rmse: 0.470428\n",
      "[2800]\tvalid_0's rmse: 0.462073\n",
      "[2900]\tvalid_0's rmse: 0.453299\n",
      "[3000]\tvalid_0's rmse: 0.444887\n",
      "Train CA_2 HOUSEHOLD_1\n"
     ]
    }
   ],
   "source": [
    "########################### Train Models\n",
    "#################################################################################\n",
    "from lightgbm import LGBMRegressor\n",
    "from gluonts.model.rotbaum._model import QRX\n",
    "for store_id in STORES:\n",
    "    for state_id in DEPTS:\n",
    "        print('Train', store_id, state_id)\n",
    "\n",
    "        grid_df, features_columns = get_data_by_store(store_id, state_id)\n",
    "\n",
    "        train_mask = grid_df['d']<=END_TRAIN\n",
    "        valid_mask = train_mask&(grid_df['d']>(END_TRAIN-P_HORIZON))\n",
    "        preds_mask = (grid_df['d']>(END_TRAIN-100)) & (grid_df['d'] <= END_TRAIN+P_HORIZON)\n",
    "\n",
    "#        train_data = lgb.Dataset(grid_df[train_mask][features_columns], \n",
    "#                           label=grid_df[train_mask][TARGET])\n",
    "\n",
    "#        valid_data = lgb.Dataset(grid_df[valid_mask][features_columns], \n",
    "#                           label=grid_df[valid_mask][TARGET])\n",
    "\n",
    "\n",
    "        seed_everything(SEED)\n",
    "        estimator = QRX(model=LGBMRegressor(**lgb_params),#lgb_wrapper(**lgb_params),\n",
    "                    min_bin_size=200)\n",
    "        estimator.fit(\n",
    "            grid_df[train_mask][features_columns], \n",
    "            grid_df[train_mask][TARGET],\n",
    "            max_sample_size=1000000, \n",
    "            seed=SEED,\n",
    "            eval_set=(\n",
    "                grid_df[valid_mask][features_columns], \n",
    "                grid_df[valid_mask][TARGET]\n",
    "            ),\n",
    "            verbose=100,\n",
    "            x_train_is_dataframe=True\n",
    "        )\n",
    "#        estimator = lgb.train(lgb_params,\n",
    "#                              train_data,\n",
    "#                              valid_sets = [valid_data],\n",
    "#                              verbose_eval = 100\n",
    "#                              \n",
    "#                              )\n",
    "        \n",
    "#        display(pd.DataFrame({'name':estimator.feature_name(),\n",
    "#                              'imp':estimator.feature_importance()}).sort_values('imp',ascending=False).head(25))\n",
    "        \n",
    "        grid_df = grid_df[preds_mask].reset_index(drop=True)\n",
    "        keep_cols = [col for col in list(grid_df) if '_tmp_' not in col]\n",
    "        grid_df = grid_df[keep_cols]\n",
    "\n",
    "        d_sales = grid_df[['d','sales']]\n",
    "        substitute = d_sales['sales'].values\n",
    "        substitute[(d_sales['d'] > END_TRAIN)] = np.nan\n",
    "        grid_df['sales'] = substitute\n",
    "\n",
    "        grid_df.to_pickle(processed_data_dir+'test_'+store_id+'_'+state_id+'.pkl')\n",
    "\n",
    "        model_name = model_dir+'lgb_model_'+store_id+'_'+state_id+'_v'+str(VER)+'.bin'\n",
    "        pickle.dump(estimator, open(model_name, 'wb'))\n",
    "\n",
    "        del grid_df, d_sales, substitute, estimator#, train_data, valid_data\n",
    "        gc.collect()\n",
    "\n",
    "        MODEL_FEATURES = features_columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "hide_input": false,
  "kernelspec": {
   "display_name": "conda_mxnet_p36",
   "language": "python",
   "name": "conda_mxnet_p36"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
